import requests
from lxml import etree
import threading
from concurrent.futures import ThreadPoolExecutor    # 导入线程池
from iteamsFile import DataFile

'''
数据请求爬虫
    传入搜索界面请求url
    调用函数
'''


"""
数据请求爬虫
    参数：
        url = str
    主要参数返回：
        self.data_list = [
            {
                "title": "",
                "img_url": "",
                "next_url": ""，
                "title_exe":"",
                "down_src":"",
            }
        ]
"""
class SpiderI:
    """
    对象初始化：
        参数：url = 数据url|str
    """
    def __init__(self,url=None,html=None):
        # 请求网址；https://flingtrainer.com/page/2/?s={search}
        self.url = url
        self.html = html

        # 请求头
        self.headers = {
            'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36',
        }

        # 全局数据初始化
        self.lock = threading.Lock()                   # 创建线程锁
        self.data_list = []                            # 数据展示列表

        # 初始化对象数据
        if url is not None:
            self.start_requests()                       # 获取初始化属性数据
        if html is not None:
            self.parse(html)


    # 发起请求
    def start_requests(self):
        response = requests.get(self.url, headers=self.headers)
        self.parse(response)

    # 数据解析
    def parse(self,response):
        tree = etree.HTML(response.text)
        lis = tree.xpath('//div[@id="content"]//article')
        for i in lis:
            # xpath遍历
            title = i.xpath('./div/h2/a/text()')
            next_url = i.xpath('.//div[@class="post-details-thumb"]/a/@href')
            img_url = i.xpath('.//div[@class="post-details-thumb"]/a/img/@src')

            data_dict = {
                'title':title[0] if title else None,
                'img_url':img_url[0] if img_url else None,
                'next_url': next_url[0] if next_url else None,
            }
            self.data_list.append(data_dict)

        self.thread_start()

    # 下载链接数据解析
    def parse_1(self,data_dict):
        if data_dict['next_url'] is not None:
            url = data_dict['next_url']
            response = requests.get(url,headers=self.headers)
            tree = etree.HTML(response.text)

            # 下载数据解析
            title = tree.xpath('//table[@class="da-attachments-table"]//td[@class="attachment-title"]/a/text()')
            src = tree.xpath('//table[@class="da-attachments-table"]//td[@class="attachment-title"]/a/@href')

            # 数据保存
            with self.lock:
                data_dict['title_exe'] = None if not title else title[0]
                data_dict['down_src'] = None if not src else src[0]
        else:
            data_dict['title_exe'] = None
            data_dict['down_src'] = None

    """
    多线程创建方法
        回调函数：def parse_1
    """
    def thread_start(self):
        with ThreadPoolExecutor(max_workers=5) as executor:
            executor.map(self.parse_1, self.data_list)

        # 等待线程执行完毕提交保存数据
        self.save()

    # 保存json数据
    def save(self):
        datafile = DataFile(self.data_list)





if __name__ == '__main__':
    spider = SpiderI('https://flingtrainer.com/?s=a')
    for data in spider.data_list:
        print(data)
